--- a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go
+++ b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go
@@ -130,16 +130,19 @@ func buildAggregateContainerStateMap(vpa *model.Vpa, cluster *model.ClusterState
 	// checkpoint to avoid having multiple peaks in the same interval after the state is restored from
 	// the checkpoint. Therefore we are extracting the current peak from all containers.
 	// TODO: Avoid the nested loop over all containers for each VPA.
-	for _, pod := range cluster.Pods {
-		for containerName, container := range pod.Containers {
-			aggregateKey := cluster.MakeAggregateStateKey(pod, containerName)
-			if vpa.UsesAggregation(aggregateKey) {
-				if aggregateContainerState, exists := aggregateContainerStateMap[containerName]; exists {
-					subtractCurrentContainerMemoryPeak(aggregateContainerState, container, now)
+	if len(vpa.ContainersInitialAggregateState) != 0 {
+		for _, pod := range cluster.Pods {
+			for containerName, container := range pod.Containers {
+				aggregateKey := cluster.MakeAggregateStateKey(pod, containerName)
+				if vpa.UsesAggregation(aggregateKey) {
+					if aggregateContainerState, exists := aggregateContainerStateMap[containerName]; exists {
+						subtractCurrentContainerMemoryPeak(aggregateContainerState, container, now)
+					}
 				}
 			}
 		}
 	}
+
 	return aggregateContainerStateMap
 }
